package com.nlp.mallet;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.corpus.tag.Nature;
import com.hankcs.hanlp.seg.Segment;
import com.hankcs.hanlp.seg.common.Term;

public class Seg {

	BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
	
	public void readLine() throws IOException {
		RandomAccessFile randomAccessFile = new RandomAccessFile("D:\\Mallet\\test\\a.txt",
				"rw");
		FileChannel channel = randomAccessFile.getChannel();
		ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024);
		int bytesRead = channel.read(buffer);
		ByteBuffer stringBuffer = ByteBuffer.allocate(20);
		while (bytesRead != -1) {
			//System.out.println("读取字节数：" + bytesRead);
			// 之前是写buffer，现在要读buffer
			buffer.flip();// 切换模式，写->读
			while (buffer.hasRemaining()) {
				byte b = buffer.get();
				if (b == 10 || b == 13) { // 换行或回车
					stringBuffer.flip();
					// 这里就是一个行
					final String line = Charset.forName("utf-8").decode(stringBuffer).toString();
					//System.out.println(line + "----------");// 解码已经读到的一行所对应的字节
					queue.offer(line);
					stringBuffer.clear();
				} else {
					if (stringBuffer.hasRemaining())
						stringBuffer.put(b);
					else { // 空间不够扩容
						stringBuffer = reAllocate(stringBuffer);
						stringBuffer.put(b);
					}
				}
			}
			buffer.clear();// 清空,position位置为0，limit=capacity
			// 继续往buffer中写
			bytesRead = channel.read(buffer);
		}
		randomAccessFile.close();
	}

	private ByteBuffer reAllocate(ByteBuffer stringBuffer) {
		final int capacity = stringBuffer.capacity();
		byte[] newBuffer = new byte[capacity * 2];
		System.arraycopy(stringBuffer.array(), 0, newBuffer, 0, capacity);
		return (ByteBuffer) ByteBuffer.wrap(newBuffer).position(capacity);
	}

	public void seg() throws FileNotFoundException {

		Segment segment = HanLP.newSegment()
		.enableNameRecognize(true)
		.enableTranslatedNameRecognize(true)
        .enableJapaneseNameRecognize(true)
        .enablePlaceRecognize(true)
        .enablePartOfSpeechTagging(true)
        .enableIndexMode(false)
        .enableNumberQuantifierRecognize(true)
        .enableCustomDictionary(false)
        .enableOffset(true)
        .enableAllNamedEntityRecognize(true)
        .enableOrganizationRecognize(true);
		
		try {
			File file = new File("D:\\Mallet\\test\\crf.model");
			//File file = new File("D:\\Mallet\\test\\sample.txt");

			
			if (file.exists()) {
				file.delete();
				try {
					file.createNewFile();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}else {
				try {
					file.createNewFile();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			Set<String> set = new HashSet<String>();
			FileOutputStream fos = new FileOutputStream(file);
			FileChannel channel = fos.getChannel();
			String reg = "([\u4e00-\\u9fa5]+)";
			while(queue.size() > 0) {
				String s = queue.take();
				List<Term> seg = segment.seg(s);
				StringBuilder sb = new StringBuilder();
				for (Term term : seg) {
					//System.out.println(term);
					Pattern pattern = Pattern.compile(reg);
					String word = term.word;
					Matcher matcher = pattern.matcher(word);
					if (matcher.find()) {
						if (set.contains(word)) {
							continue;
						}
						int frequency = term.getFrequency();
						Nature nature = term.nature;
						//System.out.println(word+" "+nature+" "+frequency);
						sb.append(word+" "+nature+" "+frequency).append("\n");
						set.add(word);
					}
				}
				if (sb.toString() == null || "".equals(sb.toString())) {
					continue;
				}
				//sb.append("\n");
				ByteBuffer buffer = ByteBuffer.allocate(sb.capacity()*2);
				buffer.put(sb.toString().getBytes());
				buffer.flip();
				try {
					channel.write(buffer);
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			try {
				fos.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
			try {
				channel.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
	
	public static void main(String[] args) throws IOException {
		
		Seg seg = new Seg();
		ExecutorService es = Executors.newFixedThreadPool(2);
		es.execute(new Runnable() {
			
			@Override
			public void run() {
				try {
					seg.readLine();
				} catch (IOException e) {
					e.printStackTrace();
				}				
			}
		});
		es.execute(new Runnable() {
			
			@Override
			public void run() {
				try {
					seg.seg();
				} catch (FileNotFoundException e) {
					e.printStackTrace();
				}
			}
		});
		
	}

}
